# -*- coding: utf-8 -*-

import os, re
from lxml import etree
from lxml.etree import XPath
from urllib.parse import quote
from spider.gonggao.GonggaoBase import GonggaoBase


'''
公告-北辰实业
'''
class Gonggao(GonggaoBase):

    def __init__(self, fmt):
        #super(Gonggao, self).__init__(fmt)  # 在python2中，需要super(自己的类名,self)来调用父类中的属性
        GonggaoBase.__init__(self, fmt)#用GonggaoBase调用__init__的方式重用父类功能

    #分页处理
    def next_page(self, url, headers, page):
        url = url % page
        htmlText = self.requestHtml('GET', url, headers)

        if htmlText==None or htmlText=='':
            exit(1)

        html = etree.HTML(htmlText)#getOffHtml(file_name)
        print(htmlText)
        try:
            host = 'https://www.aoyuan.com.cn/'
            uls_xpath = XPath("//*/ul")
            lis_xpath = XPath("li")
            time_xpath = XPath("div[@class='time']/em/text()")
            filename_xpath = XPath("div[@class='name']/text()")
            href_xpath = XPath("a/@href")
            uls = uls_xpath(html)
            for ul in uls:
                lis = lis_xpath(ul)
                for li in lis:
                    #print(etree.tostring(li, pretty_print=True).decode('utf8'))
                    try:
                        time = time_xpath(li)[0].strip()
                        filename = time + '#' + filename_xpath(li)[0].strip()
                        href = host + quote(href_xpath(li)[0].strip())
                        self.downloadFile(href, filename)
                        #print(time, filename, href)
                    except Exception as e:
                        #print(e, etree.tostring(li, pretty_print=True).decode('utf8'))
                        pass
        except Exception as e:
            print(e)

    def main(self):
        url = 'http://www.beijingns.com.cn/invest/'
        url = 'http://www.beijingns.com.cn/invest/?page=%d'
        #headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0'+str(random.randint(100, 200))}
        headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36'}
        #headers['Referer'] = 'http://ir.sohochina.com/FinancialReport.aspx'
        #headers['Cookie'] = 'ASP.NET_SessionId=lmh2ephw5lblkscrzt3hu5ii; SysLanguage=0; Hm_lvt_c5ad424ee39ca8a3459d3124e382d3b2=1620625519; Hm_lpvt_c5ad424ee39ca8a3459d3124e382d3b2=1620625519; acw_tc=65c86a0a16207991372086022ebcfd1b8d0cf74147bc75b5ca7320dd8d4804'

        company = '北辰实业'
        if not os.path.exists(company) :
            os.mkdir(company)
        os.chdir(os.path.join(os.getcwd(), company))
        for page in range(1, 2):
            self.next_page(url, headers, page)

if __name__ == '__main__':
    obj = Gonggao('%Y%m%d')
    obj.main()
